# -*- coding: utf-8 -*-
import scrapy
import time
import random

cate_info = {
    '国际': '116',
    '三农': '115',
    '搞笑': '114',
    '教育': '117',
    '文化': '112',
    '时尚': '111',
    '生活': '110',
    '情感': '109',
    '美图': '108',
    '女人': '107',
    '科学': '106',
    '科技': '105',
    '互联网': '104',
    '游戏': '15',
    '财经': '100',
    '职场': '99',
    '体育': '28',
    '社会': '27',
    '健康': '26',
    '历史': '25',
    '军事': '24',
    '房产': '22',
    '美食': '21',
    '家居': '20',
    '汽车': '19',
    '旅游': '18',
    '娱乐': '17',
    '育儿': '16',
    '宠物': '101',
    '创意': '102',
    '两性': '113',
    '辟谣': '118',
    '动漫': '103',
}
chrome = [
    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" ,
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
]

class BaiDuXinWenSpider(scrapy.Spider):
    name = 'baiduxinwen'

    def start_requests(self):
        for cate, key in cate_info.items():
            url = 'https://www.baidu.com/s?ie=utf-8&cl=2&rtt=1&bsst=1&tn=news&word={}'.format(cate)
            # url = 'https://www.baidu.com/s?tn=news&rtt=4&bsst=1&cl=2&wd={}'.format(cate) #按时间顺序排序，检索出来的百家号链接少
            yield scrapy.Request(url, callback=self.parse, headers={"User-Agent":random.choice(chrome)},meta={"cate": key})

    def parse(self, response):
        cate = response.meta["cate"]
        res_list = response.xpath('//div[contains(@class,"result")]')
        for info in res_list:
            item = {}
            item['title'] = "".join(info.xpath('.//h3[@class="c-title"]/a//text()').extract()).strip()
            item['url'] = info.xpath('.//h3/a/@href').extract_first()
            if not item["url"]:
                continue
            if 'baijiahao.baidu.com' not in item['url']:
                continue
            aa = "".join(info.xpath('.//div/p[@class="c-author"]/text()').extract()).strip().replace("\t", "").replace(
                "\n", "")
            item['author'] = aa.split('\xa0\xa0')[0]
            publish = aa.split('\xa0\xa0')[-1].strip()
            if '小时前' in publish:
                hour_num = publish.replace("小时前", "")
                now_time = int(str(time.time()).split('.')[0]) - int(hour_num) * 3600
                publish = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(now_time))
                item['publish_time'] = publish
            elif '分钟前' in publish:
                min_num = publish.replace("分钟前", "")
                now_time = int(str(time.time()).split('.')[0]) - int(min_num) * 3600
                publish = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(now_time))
                item['publish_time'] = publish
            else:
                item['publish_time'] = publish.replace("年", "-").replace("月", "-").replace("日", "") + ":00"
            item['comment_count'] = random.randint(0,500)
            item['browse_count'] = item['comment_count']*random.randint(100,1000)
            item['site_classify_id'] = cate
            item['site_id'] = 1 #1表示百家号
            # print(item)
            yield item
